package com.gs.api.reptile.sougou;


import com.alibaba.fastjson.JSONObject;
import com.gs.api.reptile.util.HttpClientUtils;
import lombok.Data;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import static com.gs.api.reptile.Constant.SOUGOU_API;

/**
 * @author guoyunfei on 2021/7/6 10:12
 * @className SougouImgProcessor
 * @description TODO
 * @Version 1.0
 */

@Data
public class SougouImgProcessor {

    private String url;
    private List<JSONObject> dataList;
    private List<String> urlList;
    private String word;

    public SougouImgProcessor(String url,String word) {
        this.url = url;
        this.word = word;
        this.dataList = new ArrayList<>();
        this.urlList = new ArrayList<>();
    }


    public void process(int idx, int size) throws IOException {
        String res = HttpClientUtils.get(String.format(this.url,idx, size,this.word));
        JSONObject object = JSONObject.parseObject(res);
        List<JSONObject> items = (List<JSONObject>)((JSONObject)object.get("data")).get("items");

        List<JSONObject> objects =  new ArrayList<>();
        for(JSONObject item : items){
            String picUrl = item.getString("picUrl");
            if(picUrl.indexOf("https") > -1){
                this.urlList.add(item.getString("picUrl"));
                objects.add(item);
            }

        }
        this.dataList.addAll(objects);
    }



    public static void main(String[] args) throws IOException {
        SougouImgProcessor processor = new SougouImgProcessor(SOUGOU_API,"小清新");

        int start = 0, size = 50, limit = 10; // 定义爬取开始索引、每次爬取数量、总共爬取数量

        for(int i=start;i<start+limit;i+=size){
            processor.process(i, size);
        }

        System.out.println(processor.urlList.toString());

        for (int i = 0; i < processor.urlList.size(); i++) {
            java.net.URL  url = new  java.net.URL(processor.urlList.get(i));

            String host = url.getHost();// 获取主机名
            System.out.println(host);
        }
    }

}
